#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
采用BeautifulSoup解析html并获取内容
注解
"""

__author__ = 'hubert'

import threading

from bs4 import BeautifulSoup
import requests
import json
import csv
import datetime
import time
import random
from multiprocessing import Pool

import logging
logging.basicConfig(level=logging.INFO)


class Movie(object):
    def __init__(self):
        # 中文名称
        self.name = ''
        # 外文名称
        self.foreignname = ''
        # 别名
        self.alias = ''
        # 导演
        self.director = ''
        # 演员
        self.actor = ''
        # 编剧
        self.screenwriter = ''
        # 出品时间/上映日期
        self.releaseyear = ''
        # 制片国家/地区
        self.area = ''
        # 类型
        self.type = ''
        # 片长
        self.duration = ''
        # 评分
        self.rate = ''
        # 简介
        self.description = ''
        # 海报地址
        self.poster_url = ''
        # 语言
        self.language = ''


# head
headers = {
    # "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.43"
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0'
}

encoding = 'uft-8'


# 把爬下来的ip地址存到文件中，调用储存在文件中的IP地址，转换为字典形式
def get_ip():
    proxies = {}
    with open('douban/ip池.txt', 'r') as r:
        datas = r.readlines()
        # 利用random随机数，随机提取IP
        data = datas[random.randint(0,11)]
        key = data[:6].strip("'")
        value = data[7:].strip("'").strip('\n').rstrip("'")
        # 转换为字典
        proxies[key] = value
        r.close()
        return proxies

proxies = {
    # "http": "http://218.14.108.53:8060"
    # "http": "http://39.99.54.91:80"
    "http": "http://47.92.113.71:80"
}


# 查看数量，用于判断循环
def douban_subjects(url):
    time.sleep(random.random()*3)
    s = requests.Session()
    r = s.get(url, headers=headers)
    # 设置编码为utf8
    r.encoding = encoding
    # 直接获取的是json串
    json_str = r.text
    # json 串，只有1个元素，key:subjects,vlaue：json串
    json_object = json.loads(json_str)
    # 数组，里面为json数据
    subjects = json_object["subjects"]
    num = len(subjects)
    return num


def get_detail_url(url):
    time.sleep(random.random()*5)
    try:
        s = requests.Session()
        r = s.get(url, headers=headers, proxies=proxies, timeout=6)
        # 设置编码为utf8
        r.encoding = encoding
    except:
        s = requests.Session()
        r = s.get(url, headers=headers, proxies=proxies, timeout=6)
        # 设置编码为utf8
        r.encoding = encoding
    # soup = BeautifulSoup(r.text, 'lxml')
    # list_wp = soup.find_all(class_="list-wp")
    # 直接获取的是json串
    json_str = r.text
    # json 串，只有1个元素，key:subjects,vlaue：json串
    # print(json_str)
    json_object = json.loads(json_str)
    # 数组，里面为json数据
    subjects = json_object["subjects"]
    list_json = subjects

    num = len(subjects)

    url_datail_list = []
    if len(list_json) < 1:
        return
    for item in list_json:
        title = item["title"]
        picurl = item["cover"]
        detail_url = item["url"]
        # print(title, picurl, detail_url)
        url_datail_list.append(detail_url)
    return url_datail_list, num


# 解析
def __parser_douban__(f, url_datail_list):
    req = requests.Session()
    for item_url in url_datail_list:

        movie = Movie()
        csv_data = []
        # 休眠，防止被禁用
        time.sleep(random.random()*3)

        # https://movie.douban.com/subject/25964071/
        res = req.get(item_url, headers=headers)
        # 设置编码为utf8
        res.encoding = 'uft-8'
        soup = BeautifulSoup(res.text, 'lxml')
        div_content = soup.find(id="content")
        dl_span = div_content.find("h1")
        titles = dl_span.find_all("span")
        # 节目名称
        movie.name = titles[0].text+titles[1].text

        print(movie.name)

        csv_data.append(movie.name)

        div_pic = div_content.find(id="mainpic")
        # 节目海报
        movie.poster_url = div_pic.find("img").get("src")
        csv_data.append(movie.poster_url)


        div_info = div_content.find(id="info")
        spans = div_info.find_all("span")

        # 导演,根据标签结构，第3个就是导演属性
        movie.director  = spans[2].text
        csv_data.append(movie.director )

        # 编剧,根据标签结构，第6个就是导演属性
        movie.screenwriter = spans[5].text
        csv_data.append(movie.screenwriter)
        # 主演，根据标签结构，第9个就是导演属性，但是又有多个span嵌套标签
        movie.actor = spans[8].text
        csv_data.append(movie.actor)

        # 类型
        type = ""
        if "类型:" == spans[9].text:
            for i in range(10, len(spans)):
                key = list(spans[i].attrs.keys())
                if "property" != key[0]:
                    break
                else:
                    if type != "":
                        type = spans[i].text + "/" + type
                    else:
                        type = spans[i].text
        movie.type = type
        csv_data.append(movie.type)

        # 制片国家/地区
        area = ''
        # 语言
        language = ''
        # 出品时间/上映日期
        releaseyear = ''
        # 片长
        duration = ''
        # 别名
        alias = ''
        for i in range(11, len(spans)):
            if "制片国家/地区:" == spans[i].text:
                # 获取标签外的文本内容，如<span></</span>123测试
                area = spans[i].next_sibling.strip()
            elif "语言:" == spans[i].text:
                language = spans[i].next_sibling.strip()
            elif "上映日期:" == spans[i].text:
                releaseyear = spans[i+1].text
            elif "片长:" == spans[i].text:
                duration = spans[i+1].text
            elif "又名:" == spans[i].text:
                alias = spans[i].next_sibling.strip()
        movie.area = area
        csv_data.append(area)
        movie.language = language
        csv_data.append(language)
        movie.releaseyear = releaseyear
        csv_data.append(releaseyear)
        movie.duration = duration
        csv_data.append(duration)
        movie.alias = alias
        csv_data.append(alias)
        # 评分
        dl_strong = div_content.find(class_="ll rating_num")
        movie.rate = dl_strong.text
        csv_data.append(movie.rate)

        # 简介
        dl_des = div_content.find(id="link-report")
        if dl_des is not None:
            movie.description = dl_des.text.strip()
        csv_data.append(movie.description)

        csv_head = []
        # 写入csv文件
        write_csv(f, csv_data)

# 写入csv文件
def write_csv(f, array_list):
    # 1. 创建文件对象
    # f = open(path,'w',encoding='utf-8')
    # 2. 基于文件对象构建 csv写入对象
    csv_writer = csv.writer(f)
    # 3. 构建列表头
    # csv_writer.writerow(["名称","code"])
    # 4. 写入csv文件内容
    csv_writer.writerow(array_list)
    # 5. 关闭文件
    # f.close()

def run_thd():
    start_time = time.time()


    end_time = time.time()
    print(end_time - start_time)

# pip安装第三方组件失败时，可采用离线安装方式
# python -m pip install lxml-4.2.3-cp37-cp37m-win_amd64.whl
if __name__ == '__main__':
    # url = 'https://movie.douban.com/subject/24773958/'
    # url = "https://movie.douban.com/explore#!type=movie&tag=%E7%83%AD%E9%97%A8&sort=rank&page_limit=20&page_start=0"
    # page_start 设置为分页limit值，即为下一页分页
    # {"subjects":[{"episodes_info":"","rate":"5.9","cover_x":1688,"title":"永恒族","url":"https:\/\/movie.douban.com\/subject\/30223888\/","playable":false,"cover":"https://img1.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2677303737.jpg","id":"30223888","cover_y":2500,"is_new":false},{"episodes_info":"","rate":"5.7","cover_x":4000,"title":"黑客帝国：矩阵重启","url":"https:\/\/movie.douban.com\/subject\/34801038\/","playable":false,"cover":"https://img3.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2844387600.jpg","id":"34801038","cover_y":5929,"is_new":false},{"episodes_info":"","rate":"6.2","cover_x":1400,"title":"精灵旅社4：变身大冒险","url":"https:\/\/movie.douban.com\/subject\/30472643\/","playable":false,"cover":"https://img3.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2659301260.jpg","id":"30472643","cover_y":2100,"is_new":true},{"episodes_info":"","rate":"6.3","cover_x":1170,"title":"奥斯维辛报告","url":"https:\/\/movie.douban.com\/subject\/34983370\/","playable":false,"cover":"https://img3.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2724272050.jpg","id":"34983370","cover_y":1728,"is_new":true},{"episodes_info":"","rate":"7.0","cover_x":800,"title":"神秘魔法部","url":"https:\/\/movie.douban.com\/subject\/35366862\/","playable":false,"cover":"https://img2.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2636895831.jpg","id":"35366862","cover_y":1200,"is_new":true},{"episodes_info":"","rate":"7.6","cover_x":1080,"title":"不要抬头","url":"https:\/\/movie.douban.com\/subject\/34884712\/","playable":false,"cover":"https://img2.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2730833093.jpg","id":"34884712","cover_y":1920,"is_new":false},{"episodes_info":"","rate":"8.7","cover_x":1000,"title":"杰伊·比姆","url":"https:\/\/movie.douban.com\/subject\/35652715\/","playable":false,"cover":"https://img2.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2734251152.jpg","id":"35652715","cover_y":1500,"is_new":false},{"episodes_info":"","rate":"6.5","cover_x":3000,"title":"魔法满屋","url":"https:\/\/movie.douban.com\/subject\/35134724\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2807936075.jpg","id":"35134724","cover_y":4188,"is_new":false},{"episodes_info":"","rate":"7.8","cover_x":2764,"title":"沙丘","url":"https:\/\/movie.douban.com\/subject\/3001114\/","playable":true,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2687443734.jpg","id":"3001114","cover_y":4096,"is_new":false},{"episodes_info":"","rate":"7.1","cover_x":1012,"title":"欢乐好声音2","url":"https:\/\/movie.douban.com\/subject\/26962981\/","playable":false,"cover":"https://img3.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2732782860.jpg","id":"26962981","cover_y":1500,"is_new":true},{"episodes_info":"","rate":"7.8","cover_x":4048,"title":"法兰西特派","url":"https:\/\/movie.douban.com\/subject\/30300279\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2634539726.jpg","id":"30300279","cover_y":6285,"is_new":false},{"episodes_info":"","rate":"6.1","cover_x":1785,"title":"古董局中局","url":"https:\/\/movie.douban.com\/subject\/26996619\/","playable":true,"cover":"https://img1.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2734316987.jpg","id":"26996619","cover_y":2500,"is_new":false},{"episodes_info":"","rate":"7.9","cover_x":595,"title":"世界上最糟糕的人","url":"https:\/\/movie.douban.com\/subject\/34447553\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2668815075.jpg","id":"34447553","cover_y":842,"is_new":false},{"episodes_info":"","rate":"8.0","cover_x":2000,"title":"驾驶我的车","url":"https:\/\/movie.douban.com\/subject\/35235502\/","playable":false,"cover":"https://img2.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2639821491.jpg","id":"35235502","cover_y":2829,"is_new":false},{"episodes_info":"","rate":"5.9","cover_x":4000,"title":"铁道英雄","url":"https:\/\/movie.douban.com\/subject\/35205446\/","playable":true,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2684720964.jpg","id":"35205446","cover_y":5600,"is_new":false},{"episodes_info":"","rate":"7.9","cover_x":2000,"title":"天鹅挽歌","url":"https:\/\/movie.douban.com\/subject\/35258381\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2717809625.jpg","id":"35258381","cover_y":3000,"is_new":false},{"episodes_info":"","rate":"7.7","cover_x":1500,"title":"犬之力","url":"https:\/\/movie.douban.com\/subject\/33437152\/","playable":false,"cover":"https://img1.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2678298618.jpg","id":"33437152","cover_y":2222,"is_new":false},{"episodes_info":"","rate":"8.3","cover_x":1500,"title":"倒数时刻","url":"https:\/\/movie.douban.com\/subject\/30279170\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2690712224.jpg","id":"30279170","cover_y":2222,"is_new":false},{"episodes_info":"","rate":"6.1","cover_x":1349,"title":"超能敢死队","url":"https:\/\/movie.douban.com\/subject\/26838236\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2685536675.jpg","id":"26838236","cover_y":2000,"is_new":false},{"episodes_info":"","rate":"8.7","cover_x":1448,"title":"花束般的恋爱","url":"https:\/\/movie.douban.com\/subject\/34874432\/","playable":false,"cover":"https://img9.doubanio.com\/view\/photo\/s_ratio_poster\/public\/p2623936924.jpg","id":"34874432","cover_y":2048,"is_new":false}]}
    # rate：评分



    limit = 20
    start_page = 0
    # 初始值
    subjects_num = 20
    # 分类
    tag = '热门'

    csv_path = "/Users/hubert/Documents/home/csv/douban_moive" + tag + datetime.datetime.now().strftime('%Y-%m-%d') + ".csv"

    # f = open(csv_path, 'w', encoding='utf-8-sig')
    with open(csv_path, 'w', encoding='utf-8-sig') as f:
        while subjects_num == 20:
            # url = "https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=" + str(limit) + "&page_start=" + str(start_page)
            url = "https://movie.douban.com/j/search_subjects?type=movie&tag=" + tag + "&sort=recommend&page_limit=" + str(limit) + "&page_start=" + str(start_page)
            # 获取每列的数量，小于20个，下次循环结束
            # subjects_num = douban_subjects(url)
            print(url)
            # 详情页URL
            result = get_detail_url(url)
            # print(result[0], result[1])
            subjects_num = result[1]
            detail_url = result[0]
            # 解析详情页
            __parser_douban__(f, detail_url)
            # 下一页
            start_page = start_page + 20
    # f.close()


